import configmap
import os
import re
import time
import urllib.request
from logutilitie import logutilitie
from publishutilitie import publishutilitie

class assassin:
    domain = ''
    decode = ''
    targetUrl = "http://www.bengou.com/folder/folderList2.jsp?letter=%s&demo=3&page=%d"
    
    letterList = ['A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z','0']
    replaceRule = """/\d*/\w*/index.html"""
    targetRule = "http://www.bengou.com/folder/folderList2.jsp\?letter=%s&demo=3&page=(?P<page>\d*)"
    downPath = os.path.join(os.getcwd(),'down/bengou/html_index')
    readUrlList = []
    #----------------------------------------------------------------------
    
    def __init__(self,domain,decode,logInputDirName):
        """Constructor"""
        
        self.domain = domain
        self.decode = decode
        print(self.decode)
        self.log = logutilitie(logInputDirName)
        if (os.path.exists(self.downPath) == False):
            os.makedirs(self.downPath)
        
        
    #----------------------------------------------------------------------
    
    def run(self,startUrl=''):
        rule4UrlList = set()
        for letter in self.letterList :
            letterIndexUrl = self.targetUrl % tuple([letter,1])
            contect = self.getHtmlFromUrl(letterIndexUrl)
            targetRule = self.targetRule % letter
            pageMax = self.getMaxPageNumber(contect,targetRule)
            for page in list(range(1,pageMax+1)):
                letterPageUrl = self.targetUrl % tuple([letter,page])
                print(letterPageUrl)
                connectPage = self.getHtmlFromUrl(letterPageUrl)
                filename = ('%s_%s.html' % tuple([letter,page]))
                self.getHtmlToFile(filename,connectPage)
                rule4UrlList = rule4UrlList.union(self.getRule4UrlList(connectPage))
        rule4UrlListLog = []
        for urle4Url in rule4UrlList :
            rule4UrlListLog = rule4UrlListLog + [("%s%s" % tuple([self.domain,urle4Url]))]
        self.log.wirteLog('as_rule_4.log',set(rule4UrlListLog))
        
                
    def getMaxPageNumber(self,contect,targetRule):
        pageList = set(re.findall(targetRule,contect))
        if(len(pageList)==0):
            page = 0
        else:
            page = max(pageList)     
        return int(page)
        
        
    def getRule4UrlList(self,contect):
        rule4UrlList = re.findall(self.replaceRule,contect)
        return set(rule4UrlList)
 
    #----------------------------------------------------------------------
    
    def getHtmlFromUrl(self,url):
        """"""
        downContent = ''
        if(url in self.readUrlList):
            return downContent
        self.readUrlList = self.readUrlList + [url]
        try:
            response = urllib.request.urlopen(url)
            downContent = response.read()
            self.log.wirteLog(__name__+'down.log',url)
        except :
            self.log.wirteLog(__name__+'error.log',url)
            return downContent
        if(self.decode):
            downContent = downContent.decode(self.decode)
        return downContent;
        
    
    #----------------------------------------------------------------------
    
    def getHtmlToFile(self,fileName,content,writeType='w'):
        """"""
        filePath = os.path.join(self.downPath,fileName)
        f=open(filePath,writeType, encoding=self.decode)
        f.write(content)
        f.close()
        return filePath
    
    #----------------------------------------------------------------------
    
    def __del__(self):
        pass
        
        
        
if(__name__=='__main__'):
    logInputDirName = input('Enter log dir name --> ')
    assassin(configmap.WebTargetUrl,configmap.WebDecode,logInputDirName).run()

